Index
#!pip install scikit-fda
import os
os.chdir("../../")
# Import libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import altair as alt
import random
import statsmodels.api as sm
from skfda.representation.grid import FDataGrid
from skfda.preprocessing.dim_reduction.projection import FPCA
from skfda.exploratory.visualization import FPCAPlot
from sklearn.preprocessing import OneHotEncoder
import skfda
from skfda.ml.regression import LinearRegression
from skfda.representation.basis import FDataBasis, FourierBasis
from skfda.exploratory.depth import IntegratedDepth, ModifiedBandDepth
from skfda.exploratory.visualization import Boxplot
# Import designed-functions
from window_extraction import calculate_window_values, calculate_window_data, Merge_data, align_to_zero, balance_index
from time_series_visualization import plot_all_time_series, plot_all_time_series_and_mean_fpca, plot_all_time_series_in_group
from functionalPCA import fpca_two_inputs, first_component_extraction, bootstrap, create_pc_scores_plots, visualize_regression
from functional_regression import Function_regression, coefficent_visualization
/var/folders/vh/dw36swbx2939r11_2dkm6r4m0000gn/T/ipykernel_50574/3302358765.py:9: DeprecationWarning: The module "projection" is deprecated. Please use "dim_reduction" from skfda.preprocessing.dim_reduction.projection import FPCA
The path of the files can be change based on where the data is stored.
# Import datasets
sensorA_System1 = pd.read_csv("../../../../RawData/System1_SensorA.csv")
sensorA_System2 = pd.read_csv("../../../../RawData/System2_SensorA.csv")
sensorB_System1 = pd.read_csv("../../../../RawData/System1_SensorB.csv")
sensorB_System2 = pd.read_csv("../../../../RawData/System2_SensorB.csv")
sensorA_System1_missing = pd.read_csv("../../../../RawData/SensorA_System1_missing values.csv")
sensorA_System2_missing = pd.read_csv("../../../../RawData/SensorA_System2_missing values.csv")
keyByTestID = pd.read_csv("../../../../RawData/Key by TestID.csv", parse_dates=['DateTime'])
# Transpose dataset to make columns as timestamps and rows as tests
# Sensor A
A1_transposed = sensorA_System1.T.reset_index()
A1_transposed.columns = A1_transposed.iloc[0]
A1_transposed.rename(columns={A1_transposed.columns[0]: 'TestID'}, inplace=True)
A1_transposed = A1_transposed.drop(0)
A1_transposed['TestID'] = A1_transposed['TestID'].astype(int)
A2_transposed = sensorA_System2.T.reset_index()
A2_transposed.columns = A2_transposed.iloc[0]
A2_transposed.rename(columns={A2_transposed.columns[0]: 'TestID'}, inplace=True)
A2_transposed = A2_transposed.drop(0)
A2_transposed['TestID'] = A2_transposed['TestID'].astype(int)
A1_missing_transposed = sensorA_System1_missing.T.reset_index()
A1_missing_transposed.columns = A1_missing_transposed.iloc[0]
A1_missing_transposed.rename(columns={A1_missing_transposed.columns[0]: 'TestID'}, inplace=True)
A1_missing_transposed = A1_missing_transposed.drop(0)
A1_missing_transposed['TestID'] = A1_missing_transposed['TestID'].astype(int)
A2_missing_transposed = sensorA_System2_missing.T.reset_index()
A2_missing_transposed.columns = A2_missing_transposed.iloc[0]
A2_missing_transposed.rename(columns={A2_missing_transposed.columns[0]: 'TestID'}, inplace=True)
A2_missing_transposed = A2_missing_transposed.drop(0)
A2_missing_transposed['TestID'] = A2_missing_transposed['TestID'].astype(int)
# Sensor B
B1_transposed = sensorB_System1.T.reset_index()
B1_transposed.columns = B1_transposed.iloc[0]
B1_transposed.rename(columns={B1_transposed.columns[0]: 'TestID'}, inplace=True)
B1_transposed = B1_transposed.drop(0)
B1_transposed['TestID'] = B1_transposed['TestID'].astype(int)
B2_transposed = sensorB_System2.T.reset_index()
B2_transposed.columns = B2_transposed.iloc[0]
B2_transposed.rename(columns={B2_transposed.columns[0]: 'TestID'}, inplace=True)
B2_transposed = B2_transposed.drop(0)
B2_transposed['TestID'] = B2_transposed['TestID'].astype(int)
# Complete A1 and A2 with the missing values
A1_transposed_mid = A1_transposed[~A1_transposed.TestID.isin(A1_missing_transposed.TestID)]
A1_transposed = pd.concat([A1_transposed_mid, A1_missing_transposed], axis=0)
A2_transposed_mid = A2_transposed[~A2_transposed.TestID.isin(A2_missing_transposed.TestID)]
A2_transposed = pd.concat([A2_transposed_mid, A2_missing_transposed], axis=0)
# Relabeling System Values
keyByTestID["System"] = keyByTestID["System"].replace({"System 2A":"System 2","System 2B":"System 2"})
# Create new column to fill fluid temperature NA's
# Note: Fluid temperature: If specified, take as the temperature of the sample fluid. The rest of the system temperature can be taken as ambient temperature.
keyByTestID['Fluid_Temperature_Filled'] = keyByTestID['Fluid Temperature'].combine_first(keyByTestID['AmbientTemperature'])
# Binning
# Categorize 'FluidType' into Blood and Aqueous
keyByTestID['FluidTypeBin'] = np.where(keyByTestID['FluidType'].str.startswith('Eurotrol'), 'Aqueous', 'Blood')
# Categorize 'AgeOfCardInDaysAtTimeOfTest' into bins
keyByTestID["CardAgeBin"] = pd.cut(keyByTestID["AgeOfCardInDaysAtTimeOfTest"], bins=[0, 9, 28, 56, 84, 112, 140, 168, 196, 224, 252],
labels=['[0-9]', '(9-28]', '(28-56]', '(56-84]', '(84-112]', '(112-140]', '(140-168]', '(168-196]', '(196-224]', '(224-252]'])
# Categorize 'Fluid_Temperature_Filled' into bins
keyByTestID["FluidTempBin"] = pd.cut(keyByTestID["Fluid_Temperature_Filled"], bins=[-1, 20, 25, 100], labels=['Below 20', '20-25', 'Above 25'])
# Filtering successful tests
keyByTestID = keyByTestID[keyByTestID['ReturnCode'].isin(['Success','UnderReportableRange'])]
# Merge dataset with keyByTestID and delete unmatched tests
keyByTestID['TestID'] = keyByTestID['TestID'].astype(int)
keyByTestID['System'] = keyByTestID['System'].astype(str)
A1_keyByTestID = keyByTestID[(keyByTestID['Sensor'] == 'Sensor A') & (keyByTestID['System'] == 'System 1')]
A1_Merged = pd.merge(A1_keyByTestID,A1_transposed,how='inner', on=['TestID'])
A1_transposed = A1_transposed[A1_transposed['TestID'].isin(A1_Merged['TestID'])]
A2_keyByTestID = keyByTestID.loc[(keyByTestID['Sensor'] == 'Sensor A') & (keyByTestID['System'] != 'System 1')]
A2_Merged = pd.merge(A2_keyByTestID,A2_transposed,how='inner', on=['TestID'])
A2_transposed = A2_transposed[A2_transposed['TestID'].isin(A2_Merged['TestID'])]
sensorA_System1 = sensorA_System1.loc[:, sensorA_System1.columns.isin(A1_Merged['TestID'].astype(str))]
sensorA_System2 = sensorA_System2.loc[:, sensorA_System2.columns.isin(A2_Merged['TestID'].astype(str))]
B1_keyByTestID = keyByTestID[(keyByTestID['Sensor'] == 'Sensor B') & (keyByTestID['System'] == 'System 1')]
B1_Merged = pd.merge(B1_keyByTestID,B1_transposed,how='inner', on=['TestID'])
B1_transposed = B1_transposed[B1_transposed['TestID'].isin(B1_Merged['TestID'])]
B2_keyByTestID = keyByTestID.loc[(keyByTestID['Sensor'] == 'Sensor B') & (keyByTestID['System'] != 'System 1')]
B2_Merged = pd.merge(B2_keyByTestID,B2_transposed,how='inner', on=['TestID'])
B1_transposed = B2_transposed[B2_transposed['TestID'].isin(A2_Merged['TestID'])]
sensorB_System1 = sensorB_System1.loc[:, sensorB_System1.columns.isin(B1_Merged['TestID'].astype(str))]
sensorB_System2 = sensorB_System2.loc[:, sensorB_System2.columns.isin(B2_Merged['TestID'].astype(str))]
print('A1: ', A1_Merged.shape)
print('A2: ', A2_Merged.shape)
print('B1: ', B1_Merged.shape)
print('B2: ', B2_Merged.shape)
A1: (3382, 3380) A2: (7743, 3371) B1: (3375, 3380) B2: (7745, 3371)
# Note: Only run once. If not, restart the kernel and run from the beggining again.
A1_Merged = A1_Merged[A1_Merged["TestID"].isin(B1_Merged["TestID"])]
B1_Merged = B1_Merged[B1_Merged["TestID"].isin(A1_Merged["TestID"])]
A2_Merged = A2_Merged[A2_Merged["TestID"].isin(B2_Merged["TestID"])]
B2_Merged = B2_Merged[B2_Merged["TestID"].isin(A2_Merged["TestID"])]
print('A1: ', A1_Merged.shape)
print('A2: ', A2_Merged.shape)
print('B1: ', B1_Merged.shape)
print('B2: ', B2_Merged.shape)
A1: (3374, 3380) A2: (7743, 3371) B1: (3374, 3380) B2: (7743, 3371)
# Match window values of Sensor A for each test
# Sensor A System 1
calDelimit = 11
cal_window_size = 8
sampleDelimit = 15
sample_window_size = 5
cal_window_start, cal_window_end, sample_window_start, sample_window_end = calculate_window_values(bubble_start=A1_Merged['BubbleDetectTime'],
sample_start=A1_Merged['SampleDetectTime'],
calDelimit_input=calDelimit,
cal_window_size_input=cal_window_size,
sampleDelimit_input=sampleDelimit,
sample_window_size_input=sample_window_size)
A1_Merged['cal_window_start']=cal_window_start
A1_Merged['cal_window_end']=cal_window_end
A1_Merged['sample_window_start']=sample_window_start
A1_Merged['sample_window_end']=sample_window_end
# Sensor A System 2
calDelimit = 9.6
cal_window_size = 8
sampleDelimit = 17.2
sample_window_size = 5
cal_window_start, cal_window_end, sample_window_start, sample_window_end = calculate_window_values(bubble_start=A2_Merged['BubbleDetectTime'],
sample_start=A2_Merged['SampleDetectTime'],
calDelimit_input=calDelimit,
cal_window_size_input=cal_window_size,
sampleDelimit_input=sampleDelimit,
sample_window_size_input=sample_window_size)
A2_Merged['cal_window_start']=cal_window_start
A2_Merged['cal_window_end']=cal_window_end
A2_Merged['sample_window_start']=sample_window_start
A2_Merged['sample_window_end']=sample_window_end
# sensor B
# For sensor B in system 1, blood and aqueous
calDelimit = 20
cal_window_size = 18
sampleDelimit_blood = 24
sampleDelimit_aqueous = 30
sample_window_size = 4
B1_Merged['cal_window_start'], B1_Merged['cal_window_end'], \
B1_Merged['sample_window_start'], B1_Merged['sample_window_end'] = zip(*B1_Merged.apply(
lambda row: calculate_window_values(
bubble_start=row['BubbleDetectTime'],
sample_start=row['SampleDetectTime'],
calDelimit_input=calDelimit,
cal_window_size_input=cal_window_size,
sampleDelimit_input=sampleDelimit_aqueous if row['FluidType'].startswith('Eurotrol') else sampleDelimit_blood,
sample_window_size_input=sample_window_size
),
axis=1
))
# For sensor B in system 2, blood and aqueous
calDelimit = 18
cal_window_size = 18
sampleDelimit_blood = 30.4
sampleDelimit_aqueous = 32.8
sample_window_size = 4
B2_Merged['cal_window_start'], B2_Merged['cal_window_end'], \
B2_Merged['sample_window_start'], B2_Merged['sample_window_end'] = zip(*B2_Merged.apply(
lambda row: calculate_window_values(
bubble_start=row['BubbleDetectTime'],
sample_start=row['SampleDetectTime'],
calDelimit_input=calDelimit,
cal_window_size_input=cal_window_size,
sampleDelimit_input=sampleDelimit_aqueous if row['FluidType'].startswith('Eurotrol') else sampleDelimit_blood,
sample_window_size_input=sample_window_size
),
axis=1
))
# Adds TestIDs as index to the values after window extraction
# System 1 - Sensor A
A1_cal_window = []
A1_sample_window = []
for i in range(len(A1_Merged)):
cal_window, sample_window = calculate_window_data(A1_Merged.iloc[i, :])
A1_cal_window.append(cal_window.values)
A1_sample_window.append(sample_window.values)
A1_cal_window = pd.DataFrame(A1_cal_window)
A1_sample_window = pd.DataFrame(A1_sample_window)
A1_cal_window['TestID'] = A1_sample_window['TestID'] = A1_Merged['TestID'].astype(int)
A1_sample_window.set_index('TestID',inplace=True)
A1_cal_window.set_index('TestID',inplace=True)
# System 2 - Sensor A
A2_cal_window = []
A2_sample_window = []
for i in range(len(A2_Merged)):
cal_window, sample_window = calculate_window_data(A2_Merged.iloc[i, :])
A2_cal_window.append(cal_window.values)
A2_sample_window.append(sample_window.values)
A2_cal_window = pd.DataFrame(A2_cal_window)
A2_sample_window = pd.DataFrame(A2_sample_window)
A2_cal_window['TestID'] = A2_sample_window['TestID'] = A2_Merged['TestID'].astype(int)
A2_sample_window.set_index('TestID',inplace=True)
A2_cal_window.set_index('TestID',inplace=True)
# System 1 - Sensor B
B1_cal_window = []
B1_sample_window = []
for i in range(len(B1_Merged)):
cal_window, sample_window = calculate_window_data(B1_Merged.iloc[i, :])
B1_cal_window.append(cal_window.values)
B1_sample_window.append(sample_window.values)
B1_cal_window = pd.DataFrame(B1_cal_window)
B1_sample_window = pd.DataFrame(B1_sample_window)
B1_cal_window['TestID'] = B1_sample_window['TestID'] = B1_Merged['TestID'].astype(int)
B1_sample_window.set_index('TestID',inplace=True)
B1_cal_window.set_index('TestID',inplace=True)
# System 2 - Sensor B
B2_cal_window = []
B2_sample_window = []
for i in range(len(B2_Merged)):
cal_window, sample_window = calculate_window_data(B2_Merged.iloc[i, :])
B2_cal_window.append(cal_window.values)
B2_sample_window.append(sample_window.values)
B2_cal_window = pd.DataFrame(B2_cal_window)
B2_sample_window = pd.DataFrame(B2_sample_window)
B2_cal_window['TestID'] = B2_sample_window['TestID'] = B2_Merged['TestID'].astype(int)
B2_sample_window.set_index('TestID',inplace=True)
B2_cal_window.set_index('TestID',inplace=True)
A1_cal_window_drop_index = A1_cal_window.loc[A1_cal_window.isna().sum(axis=1)!=0].index
A2_cal_window_drop_index = A2_cal_window.loc[A2_cal_window.isna().sum(axis=1)!=0].index
A1_sample_window_drop_index = A1_sample_window.loc[A1_sample_window.isna().sum(axis=1)!=0].index
A2_sample_window_drop_index = A2_sample_window.loc[A2_sample_window.isna().sum(axis=1)!=0].index
B1_cal_window_drop_index = B1_cal_window.loc[B1_cal_window.isna().sum(axis=1)!=0].index
B2_cal_window_drop_index = B2_cal_window.loc[B2_cal_window.isna().sum(axis=1)!=0].index
B1_sample_window_drop_index = B1_sample_window.loc[B1_sample_window.isna().sum(axis=1)!=0].index
B2_sample_window_drop_index = B2_sample_window.loc[B2_sample_window.isna().sum(axis=1)!=0].index
# Check if missing values in different windows is different
print("The missing value in calibration window:",A1_cal_window_drop_index)
print("The missing value in sample window:",A1_sample_window_drop_index)
print("The missing value in calibration window:",A2_cal_window_drop_index)
print("The missing value in sample window:",A2_sample_window_drop_index)
print("The missing value in calibration window:",B1_cal_window_drop_index)
print("The missing value in sample window:",B1_sample_window_drop_index)
print("The missing value in calibration window:",B2_cal_window_drop_index)
print("The missing value in sample window:",B2_sample_window_drop_index)
The missing value in calibration window: Float64Index([], dtype='float64', name='TestID') The missing value in sample window: Float64Index([], dtype='float64', name='TestID') The missing value in calibration window: Int64Index([], dtype='int64', name='TestID') The missing value in sample window: Int64Index([3561566, 3561567, 3565465, 3565466], dtype='int64', name='TestID') The missing value in calibration window: Float64Index([], dtype='float64', name='TestID') The missing value in sample window: Float64Index([], dtype='float64', name='TestID') The missing value in calibration window: Float64Index([], dtype='float64', name='TestID') The missing value in sample window: Float64Index([], dtype='float64', name='TestID')
# Set index for Merge datasets
A1_Merged.set_index("TestID", inplace=True)
A2_Merged.set_index("TestID", inplace=True)
B1_Merged.set_index("TestID", inplace=True)
B2_Merged.set_index("TestID", inplace=True)
# Find missing value
print("The problem indexes after extract the window are:",A1_Merged.index.difference(A1_cal_window.index))
print("The problem indexes after extract the window are:",A1_Merged.index.difference(A1_sample_window.index))
print("The problem indexes after extract the window are:",A2_Merged.index.difference(A2_cal_window.index))
print("The problem indexes after extract the window are:",A2_Merged.index.difference(A2_sample_window.index))
print("The problem indexes after extract the window are:",B1_Merged.index.difference(B1_cal_window.index))
print("The problem indexes after extract the window are:",B1_Merged.index.difference(B1_sample_window.index))
print("The problem indexes after extract the window are:",B2_Merged.index.difference(B2_cal_window.index))
print("The problem indexes after extract the window are:",B2_Merged.index.difference(B2_sample_window.index))
A1_Merged = A1_Merged.drop(A1_Merged.index.difference(A1_cal_window.index))
A1_Merged = A1_Merged.drop(A1_Merged.index.difference(A1_sample_window.index))
A2_Merged = A2_Merged.drop(A2_Merged.index.difference(A2_cal_window.index))
A2_Merged = A2_Merged.drop(A2_Merged.index.difference(A2_sample_window.index))
B1_Merged = B1_Merged.drop(B1_Merged.index.difference(B1_cal_window.index))
B1_Merged = B1_Merged.drop(B1_Merged.index.difference(B1_sample_window.index))
B2_Merged = B2_Merged.drop(B2_Merged.index.difference(B2_cal_window.index))
B2_Merged = B2_Merged.drop(B2_Merged.index.difference(B2_sample_window.index))
# Clear the Nan in index of sensor A
A1_cal_window = A1_cal_window[~A1_cal_window.index.isna()]
A1_sample_window = A1_sample_window[~A1_sample_window.index.isna()]
A2_cal_window = A2_cal_window[~A2_cal_window.index.isna()]
A2_sample_window = A2_sample_window[~A2_sample_window.index.isna()]
# Clear the Nan in index of sensor B
B1_cal_window = B1_cal_window[~B1_cal_window.index.isna()]
B1_sample_window = B1_sample_window[~B1_sample_window.index.isna()]
B2_cal_window = B2_cal_window[~B2_cal_window.index.isna()]
B2_sample_window = B2_sample_window[~B2_sample_window.index.isna()]
The problem indexes after extract the window are: Int64Index([12470355, 12470361, 12470365, 12537663, 12539049, 12622570], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([12470355, 12470361, 12470365, 12537663, 12539049, 12622570], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([12622570], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([12622570], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([3518677, 3518678], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([3518677, 3518678], dtype='int64', name='TestID')
# Shape of the subsets of time series after the extraction from the windows
# Cal Window
print('Shape of the time series after extraction')
print('A1_cal_window: ', A1_cal_window.shape)
print('A2_cal_window: ', A2_cal_window.shape)
print('B1_cal_window: ', B1_cal_window.shape)
print('B2_cal_window: ', B2_cal_window.shape)
# Sample Window
print('A1_sample_window: ', A1_sample_window.shape)
print('A2_sample_window: ', A2_sample_window.shape)
print('B1_sample_window: ', B1_sample_window.shape)
print('B2_sample_window: ', B2_sample_window.shape)
# We can delete the unmatch index but it is not necessary
Shape of the time series after extraction A1_cal_window: (3368, 41) A2_cal_window: (7743, 41) B1_cal_window: (3373, 91) B2_cal_window: (7741, 91) A1_sample_window: (3368, 26) A2_sample_window: (7743, 26) B1_sample_window: (3373, 21) B2_sample_window: (7741, 21)
# Combine data: Merge the time series with "FluidType", "AgeOfCardInDaysAtTimeOfTest", "Fluid_Temperature_Filled", "FluidTypeBin", "CardAgeBin", "FluidTempBin"
A1_cal_window_combine = Merge_data(A1_cal_window,A1_Merged)
A2_cal_window_combine = Merge_data(A2_cal_window,A2_Merged)
B1_cal_window_combine = Merge_data(B1_cal_window,B1_Merged)
B2_cal_window_combine = Merge_data(B2_cal_window,B2_Merged)
## Sample window
A1_sample_window_combine = Merge_data(A1_sample_window,A1_Merged)
A2_sample_window_combine = Merge_data(A2_sample_window,A2_Merged)
B1_sample_window_combine = Merge_data(B1_sample_window,B1_Merged)
B2_sample_window_combine = Merge_data(B2_sample_window,B2_Merged)
System1_Index, System2_Index = balance_index(A1_cal_window_combine,A2_cal_window_combine,"CardAgeBin")
System1 Sensor A & B distribution: [0-9] 142 (9-28] 142 (28-56] 142 (56-84] 142 (84-112] 142 (112-140] 142 (140-168] 142 (168-196] 142 (196-224] 142 (224-252] 142 Name: CardAgeBin, dtype: int64 System2 Sensor A & B distribution: [0-9] 142 (9-28] 142 (28-56] 142 (56-84] 142 (84-112] 142 (112-140] 142 (140-168] 142 (168-196] 142 (196-224] 142 (224-252] 142 Name: CardAgeBin, dtype: int64
# Balanced data
A1_cal_window_combine_balanced = A1_cal_window_combine.loc[System1_Index]
A1_sample_window_combine_balanced = A1_sample_window_combine.loc[System1_Index]
A2_cal_window_combine_balanced = A2_cal_window_combine.loc[System2_Index]
A2_sample_window_combine_balanced = A2_sample_window_combine.loc[System2_Index]
B1_cal_window_combine_balanced = B1_cal_window_combine.loc[System1_Index]
B1_sample_window_combine_balanced = B1_sample_window_combine.loc[System1_Index]
B2_cal_window_combine_balanced = B2_cal_window_combine.loc[System2_Index]
B2_sample_window_combine_balanced = B2_sample_window_combine.loc[System2_Index]
# Plot all the balanced time series from the window extraction
plot_all_time_series_in_group(A1_cal_window_combine_balanced, A1_sample_window_combine_balanced, A2_cal_window_combine_balanced, A2_sample_window_combine_balanced, "CardAgeBin", "System 1A - CalWindow", "System 1A - SampleWindow","System 2A - CalWindow", "System 2A - SampleWindow")
# Plot all the balanced time series from the window extraction
plot_all_time_series_in_group(B1_cal_window_combine_balanced, B1_sample_window_combine_balanced, B2_cal_window_combine_balanced, B2_sample_window_combine_balanced, "CardAgeBin", "System 1B - CalWindow", "System 1B - SampleWindow","System 2B - CalWindow", "System 2B - SampleWindow")
The following seccion will introduce
pc_scores_s1_A_cal_window, pc_scores_s2_A_cal_window,fpca_s1_A_cal_window,fpca_s2_A_cal_window = fpca_two_inputs(A1_cal_window_combine_balanced.iloc[:,:-6], A2_cal_window_combine_balanced.iloc[:,:-6], color_fpc1_s1='tab:blue', color_fpc2_s1='tab:cyan', color_fpc1_s2='tab:orange', color_fpc2_s2='gold')
print("--------------------------------------------------- Bootstrap -------------------------------------------------------------------------------------------")
ac1, ac2 = bootstrap(A1_cal_window_combine_balanced, A2_cal_window_combine_balanced,"A","cal_window",features="CardAgeBin")
print("--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------")
create_pc_scores_plots(pc_scores_s1_A_cal_window, pc_scores_s2_A_cal_window, A1_cal_window_combine_balanced, A2_cal_window_combine_balanced,features="CardAgeBin")
S1 Explain variance PC1 (%): 99.9990678032829 S1 Explain variance PC2 (%): 0.0009287030757880157 S2 Explain variance PC1 (%): 99.99922624919061 S2 Explain variance PC2 (%): 0.0007718584885239001 The time series contributing most to PC1 is at index 592 with TestID 12557583.0 The time series contributing most to PC2 is at index 800 with TestID 12529762.0 The time series contributing most to PC1 is at index 1274 with TestID 3572012 The time series contributing most to PC2 is at index 91 with TestID 3568638
/Users/dejaytang/Library/CloudStorage/OneDrive-UBC/2023mds/capstone/Capstone_Siemens/Coding/Nan/FDA_Resampling_new_window/functionalPCA.py:180: UserWarning: Attempting to set identical low and high ylims makes transformation singular; automatically expanding. plt.ylim(global_y_FPC1_min, global_y_FPC1_max)
--------------------------------------------------- Bootstrap ------------------------------------------------------------------------------------------- Confidence Interval of 1st component The number of sampling is 142 The boxplot of 1st Component
--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------
pc_scores_s1_A_sample_window, pc_scores_s2_A_sample_window,fpca_s1_A_sample_window,fpca_s2_A_sample_window = fpca_two_inputs(A1_sample_window_combine_balanced.iloc[:,:-6], A2_sample_window_combine_balanced.iloc[:,:-6], color_fpc1_s1='tab:blue', color_fpc2_s1='tab:cyan', color_fpc1_s2='tab:orange', color_fpc2_s2='gold')
print("--------------------------------------------------- Bootstrap -------------------------------------------------------------------------------------------")
as1,as2 = bootstrap(A1_sample_window_combine_balanced, A2_sample_window_combine_balanced,"A","sample_window",features="CardAgeBin")
print("--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------")
create_pc_scores_plots(pc_scores_s1_A_sample_window, pc_scores_s2_A_sample_window, A1_sample_window_combine_balanced, A2_sample_window_combine_balanced,features="CardAgeBin")
S1 Explain variance PC1 (%): 99.99971295089487 S1 Explain variance PC2 (%): 0.0002834456263569339 S2 Explain variance PC1 (%): 99.99972553144067 S2 Explain variance PC2 (%): 0.00027336964799600113 The time series contributing most to PC1 is at index 948 with TestID 12573896.0 The time series contributing most to PC2 is at index 800 with TestID 12529762.0 The time series contributing most to PC1 is at index 1152 with TestID 3572286 The time series contributing most to PC2 is at index 140 with TestID 3568703
/Users/dejaytang/Library/CloudStorage/OneDrive-UBC/2023mds/capstone/Capstone_Siemens/Coding/Nan/FDA_Resampling_new_window/functionalPCA.py:180: UserWarning: Attempting to set identical low and high ylims makes transformation singular; automatically expanding. plt.ylim(global_y_FPC1_min, global_y_FPC1_max)
--------------------------------------------------- Bootstrap ------------------------------------------------------------------------------------------- Confidence Interval of 1st component The number of sampling is 142 The boxplot of 1st Component
--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------
pc_scores_s1_B_cal_window, pc_scores_s2_B_cal_window,fpca_s1_B_cal_window,fpca_s2_B_cal_window = fpca_two_inputs(B1_cal_window_combine_balanced.iloc[:,:-6], B2_cal_window_combine_balanced.iloc[:,:-6], color_fpc1_s1='tab:blue', color_fpc2_s1='tab:cyan', color_fpc1_s2='tab:orange', color_fpc2_s2='gold')
print("--------------------------------------------------- Bootstrap -------------------------------------------------------------------------------------------")
bc1,bc2 = bootstrap(B1_cal_window_combine_balanced, B2_cal_window_combine_balanced,"B","cal_window",features="CardAgeBin")
print("--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------")
create_pc_scores_plots(pc_scores_s1_B_cal_window, pc_scores_s2_B_cal_window, B1_cal_window_combine_balanced, B2_cal_window_combine_balanced,features="CardAgeBin")
S1 Explain variance PC1 (%): 99.99240506760219 S1 Explain variance PC2 (%): 0.007565315856311003 S2 Explain variance PC1 (%): 99.9902292943407 S2 Explain variance PC2 (%): 0.009711969841695535 The time series contributing most to PC1 is at index 133 with TestID 12544066.0 The time series contributing most to PC2 is at index 82 with TestID 12615989.0 The time series contributing most to PC1 is at index 425 with TestID 3556323.0 The time series contributing most to PC2 is at index 53 with TestID 3565690.0
--------------------------------------------------- Bootstrap ------------------------------------------------------------------------------------------- Confidence Interval of 1st component The number of sampling is 142 The boxplot of 1st Component
--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------
pc_scores_s1_B_sample_window, pc_scores_s2_B_sample_window,fpca_s1_B_sample_window,fpca_s2_B_sample_window = fpca_two_inputs(B1_sample_window_combine_balanced.iloc[:,:-6], B2_sample_window_combine_balanced.iloc[:,:-6], color_fpc1_s1='tab:blue', color_fpc2_s1='tab:cyan', color_fpc1_s2='tab:orange', color_fpc2_s2='gold')
print("--------------------------------------------------- Bootstrap -------------------------------------------------------------------------------------------")
bs1,bs2 = bootstrap(B1_sample_window_combine_balanced, B2_sample_window_combine_balanced, "B","sample_window",features="CardAgeBin")
print("--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------")
create_pc_scores_plots(pc_scores_s1_B_sample_window, pc_scores_s2_B_sample_window, B1_sample_window_combine_balanced, B2_sample_window_combine_balanced,features="CardAgeBin")
S1 Explain variance PC1 (%): 99.9983476221513 S1 Explain variance PC2 (%): 0.00164163337985154 S2 Explain variance PC1 (%): 99.9989958556861 S2 Explain variance PC2 (%): 0.0009992914104035627 The time series contributing most to PC1 is at index 78 with TestID 12546583.0 The time series contributing most to PC2 is at index 684 with TestID 12191141.0 The time series contributing most to PC1 is at index 105 with TestID 3560142.0 The time series contributing most to PC2 is at index 666 with TestID 3518710.0
/Users/dejaytang/Library/CloudStorage/OneDrive-UBC/2023mds/capstone/Capstone_Siemens/Coding/Nan/FDA_Resampling_new_window/functionalPCA.py:180: UserWarning: Attempting to set identical low and high ylims makes transformation singular; automatically expanding. plt.ylim(global_y_FPC1_min, global_y_FPC1_max)
--------------------------------------------------- Bootstrap ------------------------------------------------------------------------------------------- Confidence Interval of 1st component The number of sampling is 142 The boxplot of 1st Component
--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------
df_list = []
def append_to_dataframe(window_name, slope1, slope2):
global df_list
df_list.append({'Window': window_name, 'Slope 1': slope1, 'Slope 2': slope2})
append_to_dataframe('A_cal_window', *visualize_regression(fpca_s1_A_cal_window, fpca_s2_A_cal_window))
append_to_dataframe('A_sample_window', *visualize_regression(fpca_s1_A_sample_window, fpca_s2_A_sample_window))
append_to_dataframe('B_cal_window', *visualize_regression(fpca_s1_B_cal_window, fpca_s2_B_cal_window))
append_to_dataframe('B_sample_window', *visualize_regression(fpca_s1_B_sample_window, fpca_s2_B_sample_window))
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 0.928
Model: OLS Adj. R-squared: 0.926
Method: Least Squares F-statistic: 499.0
Date: Sat, 15 Jun 2024 Prob (F-statistic): 7.84e-24
Time: 18:39:51 Log-Likelihood: 480.75
No. Observations: 41 AIC: -957.5
Df Residuals: 39 BIC: -954.1
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const -0.1581 6.15e-07 -2.57e+05 0.000 -0.158 -0.158
x1 5.913e-07 2.65e-08 22.339 0.000 5.38e-07 6.45e-07
==============================================================================
Omnibus: 4.269 Durbin-Watson: 0.346
Prob(Omnibus): 0.118 Jarque-Bera (JB): 2.726
Skew: -0.440 Prob(JB): 0.256
Kurtosis: 2.094 Cond. No. 45.7
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 0.999
Model: OLS Adj. R-squared: 0.999
Method: Least Squares F-statistic: 2.873e+04
Date: Sat, 15 Jun 2024 Prob (F-statistic): 1.52e-57
Time: 18:39:51 Log-Likelihood: 488.16
No. Observations: 41 AIC: -972.3
Df Residuals: 39 BIC: -968.9
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const -0.1582 5.13e-07 -3.08e+05 0.000 -0.158 -0.158
x1 3.745e-06 2.21e-08 169.493 0.000 3.7e-06 3.79e-06
==============================================================================
Omnibus: 10.388 Durbin-Watson: 0.250
Prob(Omnibus): 0.006 Jarque-Bera (JB): 10.067
Skew: -1.181 Prob(JB): 0.00652
Kurtosis: 3.558 Cond. No. 45.7
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 0.990
Model: OLS Adj. R-squared: 0.989
Method: Least Squares F-statistic: 2328.
Date: Sat, 15 Jun 2024 Prob (F-statistic): 2.07e-25
Time: 18:39:51 Log-Likelihood: 288.65
No. Observations: 26 AIC: -573.3
Df Residuals: 24 BIC: -570.8
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const -0.1999 1.45e-06 -1.38e+05 0.000 -0.200 -0.200
x1 -4.792e-06 9.93e-08 -48.247 0.000 -5e-06 -4.59e-06
==============================================================================
Omnibus: 1.782 Durbin-Watson: 0.222
Prob(Omnibus): 0.410 Jarque-Bera (JB): 1.587
Skew: 0.510 Prob(JB): 0.452
Kurtosis: 2.349 Cond. No. 28.4
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 1.000
Model: OLS Adj. R-squared: 1.000
Method: Least Squares F-statistic: 1.016e+05
Date: Sat, 15 Jun 2024 Prob (F-statistic): 4.85e-45
Time: 18:39:51 Log-Likelihood: 340.52
No. Observations: 26 AIC: -677.0
Df Residuals: 24 BIC: -674.5
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const -0.2001 1.97e-07 -1.02e+06 0.000 -0.200 -0.200
x1 4.307e-06 1.35e-08 318.747 0.000 4.28e-06 4.33e-06
==============================================================================
Omnibus: 1.048 Durbin-Watson: 1.238
Prob(Omnibus): 0.592 Jarque-Bera (JB): 1.025
Skew: -0.386 Prob(JB): 0.599
Kurtosis: 2.409 Cond. No. 28.4
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 1.000
Model: OLS Adj. R-squared: 1.000
Method: Least Squares F-statistic: 7.382e+05
Date: Sat, 15 Jun 2024 Prob (F-statistic): 3.45e-176
Time: 18:39:51 Log-Likelihood: 1028.7
No. Observations: 91 AIC: -2053.
Df Residuals: 89 BIC: -2048.
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const -0.1059 6.27e-07 -1.69e+05 0.000 -0.106 -0.106
x1 1.034e-05 1.2e-08 859.185 0.000 1.03e-05 1.04e-05
==============================================================================
Omnibus: 3.363 Durbin-Watson: 0.223
Prob(Omnibus): 0.186 Jarque-Bera (JB): 3.357
Skew: -0.447 Prob(JB): 0.187
Kurtosis: 2.704 Cond. No. 103.
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 1.000
Model: OLS Adj. R-squared: 1.000
Method: Least Squares F-statistic: 1.761e+06
Date: Sat, 15 Jun 2024 Prob (F-statistic): 5.46e-193
Time: 18:39:51 Log-Likelihood: 1058.9
No. Observations: 91 AIC: -2114.
Df Residuals: 89 BIC: -2109.
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const 0.1059 4.5e-07 2.36e+05 0.000 0.106 0.106
x1 -1.145e-05 8.63e-09 -1327.003 0.000 -1.15e-05 -1.14e-05
==============================================================================
Omnibus: 100.855 Durbin-Watson: 0.910
Prob(Omnibus): 0.000 Jarque-Bera (JB): 1847.334
Skew: -3.412 Prob(JB): 0.00
Kurtosis: 23.991 Cond. No. 103.
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 0.996
Model: OLS Adj. R-squared: 0.996
Method: Least Squares F-statistic: 4620.
Date: Sat, 15 Jun 2024 Prob (F-statistic): 3.76e-24
Time: 18:39:51 Log-Likelihood: 198.85
No. Observations: 21 AIC: -393.7
Df Residuals: 19 BIC: -391.6
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const -0.2231 8.28e-06 -2.7e+04 0.000 -0.223 -0.223
x1 -4.812e-05 7.08e-07 -67.971 0.000 -4.96e-05 -4.66e-05
==============================================================================
Omnibus: 2.522 Durbin-Watson: 0.137
Prob(Omnibus): 0.283 Jarque-Bera (JB): 1.955
Skew: 0.608 Prob(JB): 0.376
Kurtosis: 2.130 Cond. No. 22.7
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 0.989
Model: OLS Adj. R-squared: 0.989
Method: Least Squares F-statistic: 1756.
Date: Sat, 15 Jun 2024 Prob (F-statistic): 3.47e-20
Time: 18:39:51 Log-Likelihood: 207.27
No. Observations: 21 AIC: -410.5
Df Residuals: 19 BIC: -408.5
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const -0.2234 5.54e-06 -4.03e+04 0.000 -0.223 -0.223
x1 -1.986e-05 4.74e-07 -41.899 0.000 -2.08e-05 -1.89e-05
==============================================================================
Omnibus: 2.145 Durbin-Watson: 0.133
Prob(Omnibus): 0.342 Jarque-Bera (JB): 1.771
Skew: 0.597 Prob(JB): 0.413
Kurtosis: 2.228 Cond. No. 22.7
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
slopes_df = pd.DataFrame(df_list)
slopes_df
| Window | Slope 1 | Slope 2 | |
|---|---|---|---|
| 0 | A_cal_window | 5.913307e-07 | 0.000004 |
| 1 | A_sample_window | -4.792280e-06 | 0.000004 |
| 2 | B_cal_window | 1.033637e-05 | -0.000011 |
| 3 | B_sample_window | -4.811958e-05 | -0.000020 |
This is another functional Data Analysis method. Unlike FPCA, the following analysis utilizes the entire time series in a balanced and centered dataset as response variables for regression with the features grouped by bins. This is done to distinguish between two systems under the influence of features.
These are the coefficients from the output of the model.
print("System 1:")
A1_cal_window_funct_reg = Function_regression(A1_cal_window_combine_balanced,40,['AgeOfCardInDaysAtTimeOfTest'])
print("----------------------------------------------------------------------------")
print("\n","System 2:")
A2_cal_window_funct_reg = Function_regression(A2_cal_window_combine_balanced,40,['AgeOfCardInDaysAtTimeOfTest'])
System 1:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 40.0),), n_basis=41, period=40.0),
coefficients=[[ 1.12969934e+02 -2.54087869e-01 -6.16478393e-02 4.26477462e-02
-1.55955210e-01 -2.04932193e-01 6.98070470e-02 -1.07249592e-02
-1.97358451e-01 2.62929686e-01 6.94883436e-02 -2.02584680e-01
1.98420073e-01 -7.16210555e-02 1.68070284e-01 -4.90006365e-02
2.98174620e-01 -9.36189532e-03 2.26980579e-01 -4.54559332e-01
9.45223291e-02 1.08290355e-01 8.54976365e-02 -1.09581495e-01
1.06591476e-02 -9.73530024e-03 2.03426158e-03 -1.90255820e-01
3.99508688e-02 -4.78771163e-01 1.73005869e-01 4.45503135e-03
-3.30847265e-01 1.85403260e-01 3.05801586e-02 -2.28451401e-01
5.71844662e-02 -2.75866281e-01 7.50141766e-02 -1.85441728e+14
-4.65345409e-01]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 40.0),), n_basis=41, period=40.0),
coefficients=[[-2.63794588e+00 -5.15516832e-03 1.41558180e-03 -4.31592011e-03
-1.67517791e-03 -1.26543761e-03 -1.82353381e-04 -1.71764618e-03
-3.36598606e-03 6.41940225e-03 3.75557177e-04 2.18777832e-04
3.53878654e-03 -1.47400343e-03 -3.44271340e-04 -1.24052737e-03
3.51891726e-03 1.31600689e-03 6.31958054e-04 -1.18780137e-02
7.94026118e-04 8.87788575e-04 -3.12602046e-03 -5.49175397e-04
-7.52476552e-04 -2.30663016e-03 -1.10132979e-03 2.25616407e-04
5.52379376e-03 -7.13771225e-03 -1.83260609e-03 -2.43191772e-03
-4.99579248e-03 -4.21760416e-03 -5.85762138e-04 -1.99369278e-03
-2.38713482e-03 -5.10136445e-03 -1.45882602e-03 -3.89211385e+12
-1.09561122e-02]])
----------------------------------------------------------------------------
System 2:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 40.0),), n_basis=41, period=40.0),
coefficients=[[ 1.61580352e+02 -1.29206063e-01 2.24479908e-02 -1.56616881e-01
-1.82416575e-01 -1.07502472e-01 -5.55781743e-02 -2.42428858e-01
-1.38787426e-01 3.88194321e-01 2.48710603e-01 2.00966981e-02
2.14328148e-01 1.27789557e-02 1.56074403e-01 -4.83068955e-02
1.95708628e-01 7.04767796e-02 5.57505152e-02 -5.70402580e-01
1.55128059e-01 4.86625144e-02 -9.82674810e-02 -6.41906931e-02
-1.24195613e-01 2.09576429e-02 -4.72615716e-02 -9.77004466e-02
-5.44223014e-02 -3.64845458e-01 -3.85790740e-02 -3.16776783e-01
-3.39225282e-01 -1.32481086e-01 7.04268598e-02 -8.38290362e-02
-7.20301681e-02 -3.11544542e-01 -1.95061532e-01 -1.74444967e+14
-4.62841985e-01]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 40.0),), n_basis=41, period=40.0),
coefficients=[[-2.58267723e+00 -6.97459799e-03 1.06582977e-03 -3.54882023e-03
-1.53871235e-03 -2.39106989e-03 7.98146435e-04 -3.96808422e-04
-4.70293245e-03 6.49456462e-03 -8.47233925e-04 -1.36672920e-03
4.22802360e-03 -2.53196179e-03 -1.66368684e-04 -1.17629185e-03
4.64594195e-03 6.42731271e-04 2.55622040e-03 -1.33534625e-02
7.28062756e-04 1.39040732e-03 -2.43155637e-03 -9.51574929e-04
1.82450630e-04 -3.17648134e-03 -9.96279453e-04 -6.90144683e-04
7.03762092e-03 -9.28074329e-03 -6.21919355e-05 -5.06356522e-04
-5.84215859e-03 -2.54917080e-03 -9.08771321e-04 -4.07212747e-03
-1.74144231e-03 -5.57087047e-03 1.08385770e-03 -4.81979101e+12
-1.32473674e-02]])
print("System 1:")
A1_sample_window_funct_reg = Function_regression(A1_sample_window_combine_balanced,25,["AgeOfCardInDaysAtTimeOfTest"])
print("----------------------------------------------------------------------------")
print("\n","System 2:")
A2_sample_window_funct_reg = Function_regression(A2_sample_window_combine_balanced,25,["AgeOfCardInDaysAtTimeOfTest"])
System 1:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 25.0),), n_basis=25, period=25.0),
coefficients=[[8.61205375e+01 1.27088143e-01 1.91328493e-02 6.35335880e-02
5.06505700e-03 4.12161405e-02 2.54016366e-03 3.00481128e-02
1.01510836e-03 2.29581338e-02 9.43264471e-04 1.70733857e-02
7.30546035e-04 1.30920595e-02 4.25301550e-04 1.06563585e-02
4.68470296e-04 7.46018237e-03 5.95669041e-04 5.43748127e-03
8.96060131e-05 3.22866589e-03 4.07170310e-04 1.44224073e-03
2.19040425e-04]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 25.0),), n_basis=25, period=25.0),
coefficients=[[-1.96813078e+00 -1.81712557e-03 6.61561593e-05 -8.89158533e-04
1.63223556e-05 -5.72415295e-04 7.71483667e-06 -4.17648193e-04
6.88451146e-06 -3.16525794e-04 7.13968548e-08 -2.38887920e-04
3.88757879e-06 -1.83242005e-04 -3.46688201e-07 -1.51261251e-04
4.23082763e-06 -1.07984486e-04 2.34002348e-06 -7.25199767e-05
-4.92191161e-07 -4.70796773e-05 1.44940743e-06 -1.54952369e-05
1.33581066e-06]])
----------------------------------------------------------------------------
System 2:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 25.0),), n_basis=25, period=25.0),
coefficients=[[ 1.23255113e+02 -4.28856932e-02 -1.17243052e-02 -2.22050722e-02
-2.91034750e-03 -1.47194714e-02 -8.21504005e-04 -1.05615172e-02
-9.05009910e-04 -8.36207746e-03 -1.01059767e-03 -5.89271989e-03
-3.38205191e-04 -4.42745965e-03 3.66380096e-04 -3.57895652e-03
-5.06821488e-04 -2.44137475e-03 -6.56327343e-05 -2.14148092e-03
5.56420977e-05 -1.22036644e-03 5.27430035e-05 -8.96476133e-05
-1.55260577e-04]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 25.0),), n_basis=25, period=25.0),
coefficients=[[-1.90841233e+00 -2.78263053e-03 -5.88185442e-06 -1.36701476e-03
-3.21010779e-06 -8.88502785e-04 -4.08843089e-06 -6.37841162e-04
1.24993197e-07 -4.81239801e-04 2.96105358e-07 -3.74911049e-04
3.15311509e-07 -2.93833042e-04 -5.59362427e-06 -2.23074935e-04
9.78838012e-07 -1.67930844e-04 -1.16676654e-06 -1.14226076e-04
-1.27112858e-06 -6.54152620e-05 -3.18567396e-06 -2.52453431e-05
-1.16237400e-07]])
print("System 1:")
B1_cal_window_funct_reg = Function_regression(B1_cal_window_combine_balanced,90,["AgeOfCardInDaysAtTimeOfTest"])
print("----------------------------------------------------------------------------")
print("\n","System 2:")
B2_cal_window_funct_reg = Function_regression(B2_cal_window_combine_balanced,90,["AgeOfCardInDaysAtTimeOfTest"])
System 1:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 90.0),), n_basis=91, period=90.0),
coefficients=[[-2.65583522e+03 -3.30129041e+00 -9.30975751e+00 2.83933817e+00
-6.07673825e+00 7.82963792e+00 -2.86916728e+00 3.53985118e+00
1.18964728e+00 -3.38110939e+00 4.12124493e+00 -3.35693442e+00
-8.26391414e+00 1.22183372e+00 3.99370546e+00 -3.64919518e+00
-4.80260271e+00 -1.15372824e+00 -2.30116974e+00 3.22014780e+00
-1.46994983e+01 1.66019841e+01 1.65207352e+01 -1.31892215e+01
-1.10048244e+01 2.64561780e+00 -8.18122621e+00 9.62968584e+00
-4.40665826e+00 9.68428087e+00 -4.35548230e+00 1.46960509e+01
5.99791635e+00 -2.60883738e+00 8.27400875e+00 3.38760283e+00
1.77457892e+00 4.08949657e+00 4.78311751e+00 2.02944618e+00
4.07984148e+00 4.31230003e+00 2.59722228e-01 -7.90494435e+00
2.91802719e+00 -7.01100420e+00 -2.78555547e+00 1.10457704e+01
7.16237017e-01 5.55046550e+00 -4.85947499e+00 8.44217564e+00
3.71262914e+00 8.23434138e+00 -5.44863379e-01 1.37460693e+01
1.43268139e+01 -3.61790851e+00 1.43356239e+01 -2.88599518e+00
1.01133348e+01 -9.27232994e+00 1.15873765e+01 -4.66189277e+00
1.01565354e+01 -1.37784970e+01 7.22421391e-01 -1.05595476e+01
7.54299853e-01 -4.72556207e+00 5.21196299e+00 -7.67618593e+00
-3.96953268e+00 -8.82583750e+00 -4.24104492e-01 -8.62666925e+00
1.04774391e+00 -5.31532520e+00 -9.76943755e+00 4.19329414e+00
2.64556374e+00 -1.11608374e+01 7.15846389e-01 -6.42714208e+00
7.13414484e-01 -1.38229036e+01 -4.97710215e+00 -2.09547778e+01
7.22091356e-01 -5.70982481e+15 -3.45076589e+01]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 90.0),), n_basis=91, period=90.0),
coefficients=[[-3.31651755e-01 -2.94792336e-03 3.14315927e-03 1.62255145e-03
4.00858154e-03 -1.46635782e-02 2.37878949e-03 1.18818738e-02
1.79049083e-04 -2.02367135e-03 5.65992769e-03 2.35617973e-03
6.88204908e-04 7.05487576e-03 3.39855309e-03 5.79808060e-03
-5.52904516e-03 -4.84233616e-03 -1.21104718e-03 -4.74649217e-03
2.46276065e-04 -1.00455288e-02 3.09314123e-03 -2.57243367e-03
2.30641458e-03 4.06394024e-03 -6.91609936e-03 -3.71522473e-04
-1.22110936e-02 6.63799376e-04 4.00035835e-03 -7.06859556e-04
-3.11538868e-03 2.99058131e-03 7.05883003e-03 1.32255870e-03
2.72368613e-03 -5.14154634e-03 4.29166807e-03 -4.77162746e-03
6.61315019e-03 1.37311164e-03 2.43634741e-03 -6.36826107e-03
-3.90252021e-03 -8.40565807e-03 -3.42672286e-03 4.87598004e-03
-2.94074454e-03 -4.92381868e-03 2.99569106e-03 1.51647617e-03
7.73966547e-03 -7.88366079e-03 -2.42224671e-04 8.30337449e-04
3.91696717e-03 -2.49712154e-03 -9.21312378e-04 -6.65907289e-03
2.32340723e-03 3.73924388e-03 8.44186072e-04 -2.56339087e-03
5.02932912e-03 8.22207176e-03 -4.07845725e-03 -1.63638648e-03
2.82158791e-03 1.98338476e-03 -3.79783441e-03 -8.32099776e-04
-2.23700383e-05 3.44410157e-03 9.43511484e-04 -1.98600599e-03
-7.75107556e-03 -9.55118975e-03 -1.50231569e-04 -5.55701060e-03
-4.71877800e-03 1.51955942e-03 -4.21506964e-03 -7.75361478e-03
-2.42049600e-03 3.75292431e-03 1.12166094e-02 3.09235396e-03
-1.16694835e-02 -1.06077352e+12 -5.51165441e-03]])
----------------------------------------------------------------------------
System 2:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 90.0),), n_basis=91, period=90.0),
coefficients=[[-2.76983057e+03 -4.31500709e+00 -9.67223458e+00 3.49097739e+00
-5.93842850e+00 8.71939511e+00 -2.98438100e+00 6.25698002e+00
2.11547557e+00 -4.66772971e+00 5.50626157e+00 -3.50892156e+00
-1.01697137e+01 2.81360165e+00 4.46880900e+00 -4.39718852e+00
-7.03223746e+00 -6.56272436e+00 -1.61866902e+00 2.87378673e+00
-1.59288202e+01 2.10547047e+01 1.97207954e+01 -1.68051432e+01
-1.10545465e+01 4.09477307e+00 -9.72021804e+00 1.14942243e+01
-7.08377980e+00 1.12296107e+01 -4.88431827e+00 1.74249621e+01
6.95985038e+00 -3.25632910e+00 1.07218708e+01 3.93682029e+00
1.20978874e+00 1.82105625e+00 5.49464181e+00 1.24224558e+00
7.04872557e+00 4.95921513e+00 8.73788759e-01 -8.60693523e+00
1.27978753e+00 -8.34963157e+00 -4.32290891e+00 1.30051598e+01
2.88261168e-01 5.79441033e+00 -6.53532933e+00 1.07739464e+01
6.35418569e+00 9.98652426e+00 -5.60032045e-01 1.60533441e+01
1.75828095e+01 -3.29020431e+00 1.57108482e+01 -4.45905673e+00
1.13657593e+01 -9.92419101e+00 1.38636925e+01 -6.10233766e+00
1.22848841e+01 -1.58465041e+01 6.35491363e-01 -1.31445449e+01
1.75966152e+00 -6.44306017e+00 4.62735324e+00 -8.51874547e+00
-3.65972726e+00 -6.51985652e+00 -4.36396876e-02 -9.95893914e+00
4.01578545e+00 -7.36546572e+00 -1.05178805e+01 4.06042027e+00
3.35853929e+00 -1.32519094e+01 7.16290520e-01 -7.21288164e+00
7.32907252e-01 -1.51536817e+01 -3.59202311e+00 -2.52748117e+01
4.03959126e-01 -6.76186884e+15 -4.07699268e+01]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 90.0),), n_basis=91, period=90.0),
coefficients=[[-5.98391190e-01 2.18491315e-04 -3.60434023e-03 -1.77881688e-03
-1.39081078e-03 -1.40986809e-02 7.34088047e-04 -3.84713500e-03
-4.76958663e-03 2.70575543e-03 4.47756078e-04 -7.35511825e-04
3.43286895e-03 -3.38947748e-03 3.56564329e-03 7.46139510e-03
1.37631023e-03 3.10494633e-02 -9.59467825e-03 -1.37096154e-03
-4.17345196e-03 -2.40643412e-02 -1.43981420e-03 8.64594286e-03
-9.65913254e-03 -3.48125615e-03 -4.65897674e-03 -4.85788582e-03
6.41668957e-04 -1.42563162e-03 4.72080045e-03 -4.53964660e-03
-1.69908415e-03 5.38938723e-03 -8.12788468e-04 -2.56610309e-04
8.19341468e-03 1.26116919e-02 5.67224858e-03 2.90928423e-03
-7.97748687e-03 6.89434324e-04 -2.35403387e-03 -1.05917873e-02
7.02098906e-03 -1.18244030e-02 4.21947136e-03 2.46180286e-03
9.61363686e-04 -2.32236942e-03 1.08235082e-02 -6.49620356e-03
-7.69549381e-03 -1.32711108e-02 -1.33974098e-03 -1.79436448e-03
-3.48610872e-03 -1.38181706e-02 3.47225780e-03 3.19063124e-03
2.18607258e-03 -9.17774587e-04 -2.93935413e-03 1.81216571e-03
8.77070921e-05 9.64494755e-03 -2.75683545e-03 4.65788840e-03
-3.45611391e-03 1.07111348e-02 4.86623524e-03 -2.61001046e-03
-6.41007660e-03 -2.72123128e-02 -2.80721509e-03 -2.64963507e-03
-3.04494459e-02 -4.11233907e-03 -7.15039073e-03 -1.05205106e-03
-4.88451910e-03 2.18936934e-03 -5.05216408e-03 -9.32705018e-03
2.47522566e-04 -1.19100198e-03 -4.20959934e-04 1.15898812e-02
-1.18887521e-02 3.06775760e+11 2.04463019e-03]])
print("System 1:")
B1_sample_window_funct_reg = Function_regression(B1_sample_window_combine_balanced,20,["AgeOfCardInDaysAtTimeOfTest"])
print("----------------------------------------------------------------------------")
print("\n","System 2:")
B2_sample_window_funct_reg = Function_regression(B2_sample_window_combine_balanced,20,["AgeOfCardInDaysAtTimeOfTest"])
System 1:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 20.0),), n_basis=21, period=20.0),
coefficients=[[-1.19992659e+03 -6.14445230e-01 1.77863892e-02 -3.36699448e-01
6.05370916e-01 -3.90224307e-01 2.16845388e-01 -5.04402509e-01
1.42760995e+00 -2.19838625e+00 1.81664241e-01 -9.34748467e-02
-1.16362905e+00 -3.41843459e-01 2.06659549e-01 -3.10065648e-01
1.12370925e-01 -7.56605926e-01 1.28720853e-01 -1.01349493e+15
-1.40219641e+00]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 20.0),), n_basis=21, period=20.0),
coefficients=[[-7.79503415e-02 -5.89679082e-04 -2.66420720e-04 -3.61952375e-04
-1.58443378e-03 7.61804146e-04 -1.53152449e-03 1.09224567e-03
-2.97066289e-03 1.48850257e-03 -9.42854582e-05 -1.30842863e-03
6.10766774e-04 2.70876113e-04 -1.68424390e-03 -1.31747076e-03
-2.85573605e-03 -1.49575364e-03 -2.90301361e-04 2.79460728e+11
2.55621243e-04]])
----------------------------------------------------------------------------
System 2:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 20.0),), n_basis=21, period=20.0),
coefficients=[[-1.25306811e+03 -7.27666974e-01 -1.70689273e-01 -3.97183425e-01
2.79431449e-01 -6.48354234e-01 7.34997742e-02 -1.09444496e-01
1.21528186e+00 -2.27239358e+00 1.04118276e-01 -6.50840851e-01
-1.37728247e+00 -4.03108977e-01 -9.25808699e-02 -6.35504924e-01
-4.65317784e-01 -9.84608046e-01 -2.51478015e-01 -1.11879525e+15
-1.67383961e+00]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 20.0),), n_basis=21, period=20.0),
coefficients=[[-1.85447172e-01 -1.14637985e-04 2.58402610e-04 4.46954460e-05
5.41314550e-04 2.21372856e-03 -3.66408511e-04 -2.05667675e-03
-9.74535655e-04 9.86973823e-04 7.45545461e-04 2.52844707e-03
1.17426187e-03 8.64106434e-04 5.92008299e-04 8.34900305e-04
9.01259182e-04 -4.11051279e-05 2.47834286e-03 4.15536657e+11
1.25666894e-03]])
coefficent_visualization(A1_cal_window_funct_reg,A2_cal_window_funct_reg,["AgeOfCardInDaysAtTimeOfTest"],range(1,36),"SensorA Cal window")
coefficent_visualization(A1_sample_window_funct_reg,A2_sample_window_funct_reg,["AgeOfCardInDaysAtTimeOfTest"],range(1,23),"SensorA sample window")
coefficent_visualization(B1_cal_window_funct_reg,B2_cal_window_funct_reg,["AgeOfCardInDaysAtTimeOfTest"],range(1,86),"SensorB Cal window")
coefficent_visualization(B1_sample_window_funct_reg, B2_sample_window_funct_reg, ["AgeOfCardInDaysAtTimeOfTest"], range(1, 16), "SensorB Sample window")
